# increase width of Jupyter notebook
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:95% !important; }</style>"))
display(HTML("<style>.output_result { max-width:95% !important; }</style>"))
display(HTML("<style>.prompt { display:none !important; }</style>"))
# download data
website = "https://archive.ics.uci.edu/ml/machine-learning-databases/00286"
folder = "User Identification From Walking Activity"
url = "{}/{}.zip".format(website, folder)
url = url.replace(" ", "%20")
!curl $url -o data.zip
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 1285k 100 1285k 0 0 565k 0 0:00:02 0:00:02 --:--:-- 565k
# unzip data
from zipfile import ZipFile
with ZipFile("data.zip", 'r') as out:
out.extractall("")
# load data into pandas and visualize all of it
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import math
pd.set_option('display.max_rows', 2500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.max_colwidth', 2000)
pd.set_option('display.width', 2000)
pd.set_option('display.float_format', lambda x: '%.3f' % x)
font_size=18
params = {'legend.fontsize': 10,
'figure.figsize': (25,5),
'axes.labelsize': font_size,
'axes.titlesize': font_size,
'xtick.labelsize': font_size,
'ytick.labelsize': font_size,
'axes.titlepad': 25,
'figure.max_open_warning': False}
plt.rcParams.update(params)
number_of_users = 22
user_dataframes = {}
users_in_study = []
seconds_per_chart = 40
print("Global visualization of all data:")
for user in range(1,number_of_users+1):
csv_file = "{}/{}.csv".format(folder, user)
# publication says that "z_acceleration" should be positive when walking forward, so we switch axes ordering to make that true
walk_data = pd.read_csv(csv_file, header=None, names=['timestamp', 'x_acceleration', 'z_acceleration', 'y_acceleration'])
# precision of (x,y,z) accelerometers is high enough such that only extremely rarely should measurements be exactly identical
# however, several large blocks of data feature an anomaly where data for two axes becomes equal, so we remove this data
data_is_not_duplicated = ( (walk_data['x_acceleration'] != walk_data['y_acceleration']) &
(walk_data['y_acceleration'] != walk_data['z_acceleration']) &
(walk_data['z_acceleration'] != walk_data['x_acceleration'])
)
walk_data = walk_data[data_is_not_duplicated]
if len(walk_data.index) == 0:
print("All data for User {} is anomalous (matching sensor values): removing user from study".format(user))
else:
user_dataframes[user] = walk_data
users_in_study.append(user)
# it helps to choose a fixed scale for all data, in order to see its features across users more clearly
max_timestamp = walk_data['timestamp'].max()
charts_to_show_for_user = math.ceil(max_timestamp / seconds_per_chart)
min_timestamp_for_chart = 0
for chart in range(charts_to_show_for_user):
max_timestamp_for_chart = min_timestamp_for_chart + seconds_per_chart
chart_walk_data = walk_data[(walk_data['timestamp'] >= min_timestamp_for_chart) & (walk_data['timestamp'] <= max_timestamp_for_chart)]
walk_plot = chart_walk_data.plot( kind='line',
x='timestamp',
y='x_acceleration',
color='r',
legend=True,
title='User {}: {}-{} seconds'.format(user, int(min_timestamp_for_chart), int(max_timestamp_for_chart)),
xlim = [min_timestamp_for_chart, max_timestamp_for_chart],
ylim = [-20.0, 20.0],
yticks=[-20.0, -10.0, 0.0, 10.0, 20.0])
walk_plot = chart_walk_data.plot(kind='line', x='timestamp', y='y_acceleration', color='g', ax=walk_plot)
walk_plot = chart_walk_data.plot(kind='line', x='timestamp', y='z_acceleration', color='b', ax=walk_plot)
walk_plot.set_xlabel('time (s)')
walk_plot.set_ylabel('acceleration (m/s^2)')
min_timestamp_for_chart += seconds_per_chart
max_timestamp_for_chart += seconds_per_chart
Global visualization of all data: All data for User 19 is anomalous (matching sensor values): removing user from study
# Convert raw time-series data to a sliding window of discrete frames for a LSTM in Keras
# Filter data that is corrupt and noisy
timestamps_per_frame = 100 # approximately 3.0 seconds; hyperparameter that could be learned automatically
number_of_features_in_raw_data = 3 # x_acceleration, y_acceleration, z_acceleration
number_of_features_additionally_requested = 2 # average_acceleration, time_between_observations
number_of_features_per_timestamp = number_of_features_in_raw_data + number_of_features_additionally_requested
percent_training_data_per_user = 0.8 # we split training/testing data proportionally to number of frames per user, before combining
def shuffle_arrays_in_unison(a, b):
# used for shuffling input data and output labels with the same indexing
assert len(a) == len(b)
p = np.random.permutation(len(a))
return a[p], b[p]
def split_time_series_into_train_test_frames():
all_train_data = []
all_train_labels = []
all_test_data = []
all_test_labels = []
for user in users_in_study:
walk_data = user_dataframes[user]
number_of_timestamps = len(walk_data.index)
number_of_frames = math.ceil(number_of_timestamps / timestamps_per_frame) - 1
frames_for_user = []
min_timestamp_for_frame = 1 # always start on 2nd frame, i.e. frame 1 in frame 0, frame 1, ...
max_timestamp_for_frame = min_timestamp_for_frame + timestamps_per_frame
for frame in range(number_of_frames):
# initialize empty frame, with first dimension of 1 in preparation for stacking there
frame_data = np.zeros(shape=(1, timestamps_per_frame,number_of_features_per_timestamp), dtype=np.float32)
frame_walk_data = walk_data.iloc[min_timestamp_for_frame - 1: max_timestamp_for_frame , :]
# requested feature: time between observations => compute time difference only between each timestamp (available information at run time)
frame_timestamps = frame_walk_data["timestamp"]
frame_time_between_observations = frame_timestamps.diff().iloc[1:].values # ignore first NaN value
# as seen in graphs of data above (e.g. User 18: 0-40 seconds) sometimes past data with smaller timestamps shows up late; we ignore such data
if not np.all(frame_time_between_observations > 0.0):
#print("---> User {} data corrupted between timestamp {} - {}".format(user,min_timestamp_for_frame,max_timestamp_for_frame))
min_timestamp_for_frame += timestamps_per_frame
max_timestamp_for_frame += timestamps_per_frame
continue
# requested feature: average acceleration => average of absolute values of x,y,z for each timestamp
xyz_acceleration = frame_walk_data[['x_acceleration', 'y_acceleration', 'z_acceleration']]
average_acceleration = xyz_acceleration.abs().mean(axis=1) # absolute values highlight more vs. less motion
average_acceleration = average_acceleration.iloc[1:].values # ignore first value from previous frame
# get raw data
x_acceleration = frame_walk_data["x_acceleration"].iloc[1:].values
y_acceleration = frame_walk_data["y_acceleration"].iloc[1:].values
z_acceleration = frame_walk_data["z_acceleration"].iloc[1:].values
frame_data[0,:,0] = x_acceleration
frame_data[0,:,1] = y_acceleration
frame_data[0,:,2] = z_acceleration
frame_data[0,:,3] = average_acceleration
frame_data[0,:,4] = frame_time_between_observations
frames_for_user.append(frame_data)
min_timestamp_for_frame += timestamps_per_frame
max_timestamp_for_frame += timestamps_per_frame
frames_for_user = np.vstack(frames_for_user)
number_of_frames_for_user = frames_for_user.shape[0]
labels_for_user = np.full(shape=(number_of_frames_for_user), fill_value=user-1, dtype=np.uint16) # note: User 1 -> label 0
# shuffle data so that our training / testing split is not correlated to a specific period of data
shuffled_frames, shuffled_labels = shuffle_arrays_in_unison(frames_for_user, labels_for_user)
# by splitting training / testing data within each class, we ensure that sufficient training data for every user is encountered
max_training_frame_index = int(percent_training_data_per_user * number_of_frames_for_user)
n_train_frames = max_training_frame_index - 1
n_test_frames = number_of_frames_for_user - n_train_frames
train_data = shuffled_frames[:max_training_frame_index]
test_data = shuffled_frames[max_training_frame_index:]
train_labels = shuffled_labels[:max_training_frame_index]
test_labels = shuffled_labels[max_training_frame_index:]
print("User {}: {} frames ({} train, {} test), {} timestamps per frame, {} features per timestamp".format(user, number_of_frames_for_user, n_train_frames, n_test_frames, timestamps_per_frame, number_of_features_per_timestamp))
all_train_data.append(train_data)
all_test_data.append(test_data)
all_train_labels.append(train_labels)
all_test_labels.append(test_labels)
all_train_data = np.vstack(all_train_data)
all_test_data = np.vstack(all_test_data)
all_train_labels = np.concatenate(all_train_labels)# np.expand_dims(, axis=1)
all_test_labels = np.concatenate(all_test_labels) # np.expand_dims(, axis=1)
# finally, we shuffle the training and testing data across all users, so that right away our network is training / testing across all users
train_X, train_Y = shuffle_arrays_in_unison(all_train_data, all_train_labels)
test_X, test_Y = shuffle_arrays_in_unison(all_test_data, all_test_labels)
print("\nShape of training data: {}".format(train_X.shape))
print("Shape of training labels: {}".format(train_Y.shape))
print("Shape of testing data: {}".format(test_X.shape))
print("Shape of testing labels: {}".format(test_Y.shape))
return train_X, test_X, train_Y, test_Y
train_X, test_X, train_Y, test_Y = split_time_series_into_train_test_frames()
User 1: 50 frames (39 train, 11 test), 100 timestamps per frame, 5 features per timestamp User 2: 38 frames (29 train, 9 test), 100 timestamps per frame, 5 features per timestamp User 3: 11 frames (7 train, 4 test), 100 timestamps per frame, 5 features per timestamp User 4: 69 frames (54 train, 15 test), 100 timestamps per frame, 5 features per timestamp User 5: 11 frames (7 train, 4 test), 100 timestamps per frame, 5 features per timestamp User 6: 49 frames (38 train, 11 test), 100 timestamps per frame, 5 features per timestamp User 7: 37 frames (28 train, 9 test), 100 timestamps per frame, 5 features per timestamp User 8: 34 frames (26 train, 8 test), 100 timestamps per frame, 5 features per timestamp User 9: 79 frames (62 train, 17 test), 100 timestamps per frame, 5 features per timestamp User 10: 30 frames (23 train, 7 test), 100 timestamps per frame, 5 features per timestamp User 11: 56 frames (43 train, 13 test), 100 timestamps per frame, 5 features per timestamp User 12: 47 frames (36 train, 11 test), 100 timestamps per frame, 5 features per timestamp User 13: 66 frames (51 train, 15 test), 100 timestamps per frame, 5 features per timestamp User 14: 120 frames (95 train, 25 test), 100 timestamps per frame, 5 features per timestamp User 15: 36 frames (27 train, 9 test), 100 timestamps per frame, 5 features per timestamp User 16: 17 frames (12 train, 5 test), 100 timestamps per frame, 5 features per timestamp User 17: 216 frames (171 train, 45 test), 100 timestamps per frame, 5 features per timestamp User 18: 187 frames (148 train, 39 test), 100 timestamps per frame, 5 features per timestamp User 20: 167 frames (132 train, 35 test), 100 timestamps per frame, 5 features per timestamp User 21: 30 frames (23 train, 7 test), 100 timestamps per frame, 5 features per timestamp User 22: 96 frames (75 train, 21 test), 100 timestamps per frame, 5 features per timestamp Shape of training data: (1147, 100, 5) Shape of training labels: (1147,) Shape of testing data: (299, 100, 5) Shape of testing labels: (299,)
from keras.models import Sequential, load_model
from keras.layers import Dense, Flatten, Dropout, LSTM
from keras.utils import to_categorical
# convert integer labels to one-hot encoded vector, so final embedding layer can predict distributions over users
one_hot_train_Y = to_categorical(train_Y)
one_hot_test_Y = to_categorical(test_Y)
print("Shape of train labels after one-hot encoding: {}".format(one_hot_train_Y.shape))
print("Shape of test labels after one-hot encoding: {}".format(one_hot_test_Y.shape))
# arbitrary LSTM neural network architecture inspired by https://machinelearningmastery.com/how-to-develop-rnn-models-for-human-activity-recognition-time-series-classification/
# for best performance on a real dataset, we could easily automate a search for types of layers, number of layers, hyperparameters of layers, types of activations, ...
number_of_lstm_parameters = 125
probability_of_dropout = 0.25
number_of_dense_parameters = 250
type_of_dense_activation = 'relu'
type_of_loss_function = 'categorical_crossentropy'
type_of_final_activation = 'softmax'
optimizer = 'adam'
metrics = ['accuracy']
input_batch_size = 64 # could be optimized for hardware
epochs_between_tests = 20
number_of_models_to_test = 15
# Define vanilla LSTM architecture
model = Sequential()
model.add(LSTM(number_of_lstm_parameters, input_shape=(timestamps_per_frame, number_of_features_per_timestamp)))
model.add(Dropout(1-probability_of_dropout))
model.add(Dense(number_of_dense_parameters, activation=type_of_dense_activation))
model.add(Dense(number_of_users, activation=type_of_final_activation))
model.compile(loss=type_of_loss_function, optimizer=optimizer, metrics=metrics)
model.summary()
all_test_model_accuracies = np.zeros(number_of_models_to_test, dtype=np.float32)
model_filepaths = []
epochs_trained = []
for testing_index in range(number_of_models_to_test):
model.fit(train_X, one_hot_train_Y, epochs=epochs_between_tests, batch_size=input_batch_size)
_, accuracy = model.evaluate(test_X, one_hot_test_Y, batch_size=one_hot_test_Y.shape[0])
number_of_epochs_trained_for_model = (testing_index+1) * epochs_between_tests
epochs_trained.append(number_of_epochs_trained_for_model)
print("Testing accuracy after {} epochs: {:.1f}%\n\n".format(number_of_epochs_trained_for_model, 100*accuracy))
all_test_model_accuracies[testing_index] = accuracy
model_filepath = "{}/lstm_walker_recognition_model_{}.h5".format(folder, testing_index)
model.save(model_filepath)
model_filepaths.append(model_filepath)
max_model_index = np.argmax(all_test_model_accuracies)
best_model_filepath = model_filepaths[max_model_index]
best_model_test_accuracy = all_test_model_accuracies[max_model_index]
best_model_epochs_trained = epochs_trained[max_model_index]
print("\nThe model trained for {} epochs performed best on held out test data ({:.1f}% accuracy)".format(best_model_epochs_trained, 100*best_model_test_accuracy))
Shape of train labels after one-hot encoding: (1147, 22) Shape of test labels after one-hot encoding: (299, 22) Model: "sequential_30" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= lstm_29 (LSTM) (None, 125) 65500 _________________________________________________________________ dropout_28 (Dropout) (None, 125) 0 _________________________________________________________________ dense_56 (Dense) (None, 250) 31500 _________________________________________________________________ dense_57 (Dense) (None, 22) 5522 ================================================================= Total params: 102,522 Trainable params: 102,522 Non-trainable params: 0 _________________________________________________________________ Epoch 1/20 18/18 [==============================] - 2s 116ms/step - loss: 2.8804 - accuracy: 0.1526 Epoch 2/20 18/18 [==============================] - 2s 122ms/step - loss: 2.5227 - accuracy: 0.2616 Epoch 3/20 18/18 [==============================] - 2s 122ms/step - loss: 2.2776 - accuracy: 0.2877 Epoch 4/20 18/18 [==============================] - 2s 116ms/step - loss: 2.1338 - accuracy: 0.3191 Epoch 5/20 18/18 [==============================] - 2s 120ms/step - loss: 2.0112 - accuracy: 0.3496 Epoch 6/20 18/18 [==============================] - 2s 119ms/step - loss: 2.0286 - accuracy: 0.3235 Epoch 7/20 18/18 [==============================] - 2s 127ms/step - loss: 1.8577 - accuracy: 0.3827 Epoch 8/20 18/18 [==============================] - 2s 116ms/step - loss: 1.7783 - accuracy: 0.3923 Epoch 9/20 18/18 [==============================] - 2s 116ms/step - loss: 1.7102 - accuracy: 0.4037 Epoch 10/20 18/18 [==============================] - 2s 118ms/step - loss: 1.6949 - accuracy: 0.4202 Epoch 11/20 18/18 [==============================] - 2s 119ms/step - loss: 1.7130 - accuracy: 0.4080 Epoch 12/20 18/18 [==============================] - 2s 130ms/step - loss: 1.7422 - accuracy: 0.3967 Epoch 13/20 18/18 [==============================] - 2s 129ms/step - loss: 1.8473 - accuracy: 0.3810 Epoch 14/20 18/18 [==============================] - 2s 127ms/step - loss: 1.7116 - accuracy: 0.4167 Epoch 15/20 18/18 [==============================] - 2s 135ms/step - loss: 1.7087 - accuracy: 0.4080 Epoch 16/20 18/18 [==============================] - 2s 128ms/step - loss: 1.5921 - accuracy: 0.4429 Epoch 17/20 18/18 [==============================] - 2s 115ms/step - loss: 1.5341 - accuracy: 0.4664 Epoch 18/20 18/18 [==============================] - 2s 114ms/step - loss: 1.5178 - accuracy: 0.4647 Epoch 19/20 18/18 [==============================] - 2s 116ms/step - loss: 1.4606 - accuracy: 0.4935 Epoch 20/20 18/18 [==============================] - 2s 117ms/step - loss: 1.5121 - accuracy: 0.4690 1/1 [==============================] - 0s 2ms/step - loss: 1.4340 - accuracy: 0.5385 Testing accuracy after 20 epochs: 53.8% Epoch 1/20 18/18 [==============================] - 2s 133ms/step - loss: 1.4814 - accuracy: 0.4699 Epoch 2/20 18/18 [==============================] - 2s 123ms/step - loss: 1.4364 - accuracy: 0.5109 Epoch 3/20 18/18 [==============================] - 2s 116ms/step - loss: 1.4365 - accuracy: 0.4969 Epoch 4/20 18/18 [==============================] - 2s 124ms/step - loss: 1.4799 - accuracy: 0.4542 Epoch 5/20 18/18 [==============================] - 2s 120ms/step - loss: 1.4625 - accuracy: 0.4856 Epoch 6/20 18/18 [==============================] - 2s 120ms/step - loss: 1.4466 - accuracy: 0.4795 Epoch 7/20 18/18 [==============================] - 2s 117ms/step - loss: 1.4338 - accuracy: 0.4891 Epoch 8/20 18/18 [==============================] - 2s 118ms/step - loss: 1.4145 - accuracy: 0.4882 Epoch 9/20 18/18 [==============================] - 2s 116ms/step - loss: 1.4198 - accuracy: 0.4978 Epoch 10/20 18/18 [==============================] - 2s 116ms/step - loss: 1.3893 - accuracy: 0.5048 Epoch 11/20 18/18 [==============================] - 2s 116ms/step - loss: 1.3607 - accuracy: 0.5318 Epoch 12/20 18/18 [==============================] - 2s 123ms/step - loss: 1.3256 - accuracy: 0.5336 Epoch 13/20 18/18 [==============================] - 2s 116ms/step - loss: 1.3083 - accuracy: 0.5371 Epoch 14/20 18/18 [==============================] - 2s 116ms/step - loss: 1.3489 - accuracy: 0.5231 Epoch 15/20 18/18 [==============================] - 2s 117ms/step - loss: 1.3359 - accuracy: 0.5458 Epoch 16/20 18/18 [==============================] - 2s 117ms/step - loss: 1.2681 - accuracy: 0.5519 Epoch 17/20 18/18 [==============================] - 2s 117ms/step - loss: 1.2708 - accuracy: 0.5501 Epoch 18/20 18/18 [==============================] - 2s 119ms/step - loss: 1.2269 - accuracy: 0.5876 Epoch 19/20 18/18 [==============================] - 2s 116ms/step - loss: 1.2498 - accuracy: 0.5606 Epoch 20/20 18/18 [==============================] - 2s 117ms/step - loss: 1.3828 - accuracy: 0.5458 1/1 [==============================] - 0s 1ms/step - loss: 1.6178 - accuracy: 0.4649 Testing accuracy after 40 epochs: 46.5% Epoch 1/20 18/18 [==============================] - 2s 118ms/step - loss: 1.4958 - accuracy: 0.4743 Epoch 2/20 18/18 [==============================] - 2s 117ms/step - loss: 1.3902 - accuracy: 0.4969 Epoch 3/20 18/18 [==============================] - 2s 123ms/step - loss: 1.2799 - accuracy: 0.5484 Epoch 4/20 18/18 [==============================] - 2s 116ms/step - loss: 1.2314 - accuracy: 0.5493 Epoch 5/20 18/18 [==============================] - 2s 115ms/step - loss: 1.2320 - accuracy: 0.5588 Epoch 6/20 18/18 [==============================] - 2s 116ms/step - loss: 1.2302 - accuracy: 0.5545 Epoch 7/20 18/18 [==============================] - 2s 116ms/step - loss: 1.2378 - accuracy: 0.5536 Epoch 8/20 18/18 [==============================] - 2s 117ms/step - loss: 1.2520 - accuracy: 0.5606 Epoch 9/20 18/18 [==============================] - 2s 116ms/step - loss: 1.2027 - accuracy: 0.5693 Epoch 10/20 18/18 [==============================] - 2s 117ms/step - loss: 1.1964 - accuracy: 0.5702 Epoch 11/20 18/18 [==============================] - 2s 117ms/step - loss: 1.1682 - accuracy: 0.5920 Epoch 12/20 18/18 [==============================] - 2s 117ms/step - loss: 1.1492 - accuracy: 0.5920 Epoch 13/20 18/18 [==============================] - 2s 119ms/step - loss: 1.1960 - accuracy: 0.5876 Epoch 14/20 18/18 [==============================] - 2s 116ms/step - loss: 1.1836 - accuracy: 0.5745 Epoch 15/20 18/18 [==============================] - 2s 114ms/step - loss: 1.1256 - accuracy: 0.6120 Epoch 16/20 18/18 [==============================] - 2s 114ms/step - loss: 1.1420 - accuracy: 0.6007 Epoch 17/20 18/18 [==============================] - 2s 115ms/step - loss: 1.1130 - accuracy: 0.6103 Epoch 18/20 18/18 [==============================] - 2s 115ms/step - loss: 1.1448 - accuracy: 0.6251 Epoch 19/20 18/18 [==============================] - 2s 118ms/step - loss: 1.1260 - accuracy: 0.6068 Epoch 20/20 18/18 [==============================] - 2s 118ms/step - loss: 1.0653 - accuracy: 0.6216 1/1 [==============================] - 0s 2ms/step - loss: 1.2213 - accuracy: 0.6187 Testing accuracy after 60 epochs: 61.9% Epoch 1/20 18/18 [==============================] - 2s 118ms/step - loss: 1.0771 - accuracy: 0.6234 Epoch 2/20 18/18 [==============================] - 2s 117ms/step - loss: 1.1331 - accuracy: 0.5902 Epoch 3/20 18/18 [==============================] - 2s 117ms/step - loss: 1.0763 - accuracy: 0.6286 Epoch 4/20 18/18 [==============================] - 2s 119ms/step - loss: 1.0961 - accuracy: 0.6155 Epoch 5/20 18/18 [==============================] - 2s 116ms/step - loss: 1.1108 - accuracy: 0.6024 Epoch 6/20 18/18 [==============================] - 2s 117ms/step - loss: 1.0310 - accuracy: 0.6399 Epoch 7/20 18/18 [==============================] - 2s 116ms/step - loss: 1.0109 - accuracy: 0.6548 Epoch 8/20 18/18 [==============================] - 2s 114ms/step - loss: 1.1044 - accuracy: 0.6077 Epoch 9/20 18/18 [==============================] - 2s 113ms/step - loss: 1.1281 - accuracy: 0.5946 Epoch 10/20 18/18 [==============================] - 2s 111ms/step - loss: 1.0738 - accuracy: 0.6234 Epoch 11/20 18/18 [==============================] - 2s 111ms/step - loss: 1.0525 - accuracy: 0.6251 Epoch 12/20 18/18 [==============================] - 2s 118ms/step - loss: 1.0372 - accuracy: 0.6443 Epoch 13/20 18/18 [==============================] - 3s 145ms/step - loss: 1.0216 - accuracy: 0.6356 Epoch 14/20 18/18 [==============================] - 2s 119ms/step - loss: 0.9470 - accuracy: 0.6565 Epoch 15/20 18/18 [==============================] - 2s 115ms/step - loss: 0.9538 - accuracy: 0.6565 Epoch 16/20 18/18 [==============================] - 2s 113ms/step - loss: 0.9431 - accuracy: 0.6609 Epoch 17/20 18/18 [==============================] - 2s 115ms/step - loss: 1.0047 - accuracy: 0.6417 Epoch 18/20 18/18 [==============================] - 2s 114ms/step - loss: 0.9713 - accuracy: 0.6495 Epoch 19/20 18/18 [==============================] - 2s 114ms/step - loss: 0.9290 - accuracy: 0.6696 Epoch 20/20 18/18 [==============================] - 2s 113ms/step - loss: 0.9464 - accuracy: 0.6678 1/1 [==============================] - 0s 1ms/step - loss: 1.2648 - accuracy: 0.6154 Testing accuracy after 80 epochs: 61.5% Epoch 1/20 18/18 [==============================] - 2s 115ms/step - loss: 0.9574 - accuracy: 0.6478 Epoch 2/20 18/18 [==============================] - 2s 114ms/step - loss: 0.9215 - accuracy: 0.6827 Epoch 3/20 18/18 [==============================] - 2s 114ms/step - loss: 0.9637 - accuracy: 0.6504 Epoch 4/20 18/18 [==============================] - 2s 114ms/step - loss: 0.9257 - accuracy: 0.6800 Epoch 5/20 18/18 [==============================] - 2s 112ms/step - loss: 0.9147 - accuracy: 0.6626 Epoch 6/20 18/18 [==============================] - 2s 113ms/step - loss: 0.8815 - accuracy: 0.6975 Epoch 7/20 18/18 [==============================] - 2s 114ms/step - loss: 0.8997 - accuracy: 0.6853 Epoch 8/20 18/18 [==============================] - 2s 112ms/step - loss: 0.9143 - accuracy: 0.6818 Epoch 9/20 18/18 [==============================] - 2s 112ms/step - loss: 0.8464 - accuracy: 0.6949 Epoch 10/20 18/18 [==============================] - 2s 115ms/step - loss: 0.8486 - accuracy: 0.7036 Epoch 11/20 18/18 [==============================] - 2s 113ms/step - loss: 0.7801 - accuracy: 0.7149 Epoch 12/20 18/18 [==============================] - 2s 113ms/step - loss: 0.9377 - accuracy: 0.6696 Epoch 13/20 18/18 [==============================] - 2s 113ms/step - loss: 0.8396 - accuracy: 0.6914 Epoch 14/20 18/18 [==============================] - 2s 117ms/step - loss: 0.7966 - accuracy: 0.7062 Epoch 15/20 18/18 [==============================] - 2s 131ms/step - loss: 0.8088 - accuracy: 0.7114 Epoch 16/20 18/18 [==============================] - 2s 114ms/step - loss: 0.7730 - accuracy: 0.7184 Epoch 17/20 18/18 [==============================] - 2s 114ms/step - loss: 0.7844 - accuracy: 0.7193 Epoch 18/20 18/18 [==============================] - 2s 114ms/step - loss: 0.7918 - accuracy: 0.7201 Epoch 19/20 18/18 [==============================] - 2s 114ms/step - loss: 0.8333 - accuracy: 0.6931 Epoch 20/20 18/18 [==============================] - 2s 114ms/step - loss: 0.8186 - accuracy: 0.7001 1/1 [==============================] - 0s 1ms/step - loss: 1.1761 - accuracy: 0.6455 Testing accuracy after 100 epochs: 64.5% Epoch 1/20 18/18 [==============================] - 2s 121ms/step - loss: 0.8529 - accuracy: 0.7079 Epoch 2/20 18/18 [==============================] - 2s 112ms/step - loss: 0.8439 - accuracy: 0.7053 Epoch 3/20 18/18 [==============================] - 2s 113ms/step - loss: 0.8658 - accuracy: 0.6879 Epoch 4/20 18/18 [==============================] - 2s 114ms/step - loss: 0.8081 - accuracy: 0.7201 Epoch 5/20 18/18 [==============================] - 2s 115ms/step - loss: 0.8553 - accuracy: 0.7044 Epoch 6/20 18/18 [==============================] - 2s 113ms/step - loss: 0.8519 - accuracy: 0.6940 Epoch 7/20 18/18 [==============================] - 2s 113ms/step - loss: 0.8177 - accuracy: 0.7271 Epoch 8/20 18/18 [==============================] - 2s 116ms/step - loss: 0.7690 - accuracy: 0.7228 Epoch 9/20 18/18 [==============================] - 2s 113ms/step - loss: 0.7694 - accuracy: 0.7201 Epoch 10/20 18/18 [==============================] - 2s 113ms/step - loss: 0.6989 - accuracy: 0.7568 Epoch 11/20 18/18 [==============================] - 2s 113ms/step - loss: 0.7066 - accuracy: 0.7550 Epoch 12/20 18/18 [==============================] - 2s 121ms/step - loss: 0.6713 - accuracy: 0.7681 Epoch 13/20 18/18 [==============================] - 2s 114ms/step - loss: 0.6908 - accuracy: 0.7759 Epoch 14/20 18/18 [==============================] - 2s 112ms/step - loss: 0.6874 - accuracy: 0.7751 Epoch 15/20 18/18 [==============================] - 2s 113ms/step - loss: 0.7797 - accuracy: 0.7428 Epoch 16/20 18/18 [==============================] - 2s 112ms/step - loss: 0.9221 - accuracy: 0.6931 Epoch 17/20 18/18 [==============================] - 2s 114ms/step - loss: 0.8865 - accuracy: 0.7149 Epoch 18/20 18/18 [==============================] - 2s 114ms/step - loss: 0.7842 - accuracy: 0.7524 Epoch 19/20 18/18 [==============================] - 2s 114ms/step - loss: 0.7710 - accuracy: 0.7350 Epoch 20/20 18/18 [==============================] - 2s 113ms/step - loss: 0.7329 - accuracy: 0.7568 1/1 [==============================] - 0s 1ms/step - loss: 1.1685 - accuracy: 0.6555 Testing accuracy after 120 epochs: 65.6% Epoch 1/20 18/18 [==============================] - 2s 114ms/step - loss: 0.7419 - accuracy: 0.7376 Epoch 2/20 18/18 [==============================] - 2s 113ms/step - loss: 0.7236 - accuracy: 0.7620 Epoch 3/20 18/18 [==============================] - 2s 122ms/step - loss: 0.7457 - accuracy: 0.7446 Epoch 4/20 18/18 [==============================] - 2s 113ms/step - loss: 0.6558 - accuracy: 0.7847 Epoch 5/20 18/18 [==============================] - 2s 114ms/step - loss: 0.6632 - accuracy: 0.7611 Epoch 6/20 18/18 [==============================] - 2s 113ms/step - loss: 0.6546 - accuracy: 0.7751 Epoch 7/20 18/18 [==============================] - 3s 140ms/step - loss: 0.6437 - accuracy: 0.7812 Epoch 8/20 18/18 [==============================] - 2s 121ms/step - loss: 0.6316 - accuracy: 0.7995 Epoch 9/20 18/18 [==============================] - 2s 124ms/step - loss: 0.6159 - accuracy: 0.7916 Epoch 10/20 18/18 [==============================] - 2s 136ms/step - loss: 0.6638 - accuracy: 0.7777 Epoch 11/20 18/18 [==============================] - 3s 148ms/step - loss: 0.6378 - accuracy: 0.7873 Epoch 12/20 18/18 [==============================] - 3s 159ms/step - loss: 0.6927 - accuracy: 0.7620 Epoch 13/20 18/18 [==============================] - 3s 163ms/step - loss: 0.6344 - accuracy: 0.7786 Epoch 14/20 18/18 [==============================] - 3s 140ms/step - loss: 0.5819 - accuracy: 0.8012 Epoch 15/20 18/18 [==============================] - 3s 161ms/step - loss: 0.7403 - accuracy: 0.7768 Epoch 16/20 18/18 [==============================] - 3s 160ms/step - loss: 0.7081 - accuracy: 0.7533 Epoch 17/20 18/18 [==============================] - 3s 164ms/step - loss: 0.7702 - accuracy: 0.7498 Epoch 18/20 18/18 [==============================] - 3s 154ms/step - loss: 0.7069 - accuracy: 0.7777 Epoch 19/20 18/18 [==============================] - 2s 130ms/step - loss: 0.6537 - accuracy: 0.7672 Epoch 20/20 18/18 [==============================] - 3s 154ms/step - loss: 0.6492 - accuracy: 0.7942 1/1 [==============================] - 0s 3ms/step - loss: 1.0967 - accuracy: 0.6957 Testing accuracy after 140 epochs: 69.6% Epoch 1/20 18/18 [==============================] - 3s 153ms/step - loss: 0.6606 - accuracy: 0.7881 Epoch 2/20 18/18 [==============================] - 2s 116ms/step - loss: 0.6430 - accuracy: 0.7838 Epoch 3/20 18/18 [==============================] - 2s 139ms/step - loss: 0.5583 - accuracy: 0.8152 Epoch 4/20 18/18 [==============================] - 2s 138ms/step - loss: 0.5465 - accuracy: 0.8152 Epoch 5/20 18/18 [==============================] - 3s 145ms/step - loss: 0.5019 - accuracy: 0.8352 Epoch 6/20 18/18 [==============================] - 3s 140ms/step - loss: 0.5166 - accuracy: 0.8335 Epoch 7/20 18/18 [==============================] - 3s 150ms/step - loss: 0.6125 - accuracy: 0.7995 Epoch 8/20 18/18 [==============================] - 2s 114ms/step - loss: 0.6203 - accuracy: 0.7899 Epoch 9/20 18/18 [==============================] - 2s 112ms/step - loss: 0.5901 - accuracy: 0.8073 Epoch 10/20 18/18 [==============================] - 2s 113ms/step - loss: 0.5595 - accuracy: 0.8152 Epoch 11/20 18/18 [==============================] - 2s 118ms/step - loss: 0.6216 - accuracy: 0.8003 Epoch 12/20 18/18 [==============================] - 2s 115ms/step - loss: 0.6198 - accuracy: 0.8003 Epoch 13/20 18/18 [==============================] - 2s 112ms/step - loss: 0.5682 - accuracy: 0.8047 Epoch 14/20 18/18 [==============================] - 2s 113ms/step - loss: 0.5344 - accuracy: 0.8282 Epoch 15/20 18/18 [==============================] - 2s 113ms/step - loss: 0.4786 - accuracy: 0.8361 Epoch 16/20 18/18 [==============================] - 2s 120ms/step - loss: 0.4764 - accuracy: 0.8535 Epoch 17/20 18/18 [==============================] - 2s 112ms/step - loss: 0.4602 - accuracy: 0.8649 Epoch 18/20 18/18 [==============================] - 2s 113ms/step - loss: 0.4927 - accuracy: 0.8483 Epoch 19/20 18/18 [==============================] - 2s 113ms/step - loss: 0.5662 - accuracy: 0.8230 Epoch 20/20 18/18 [==============================] - 2s 113ms/step - loss: 0.5973 - accuracy: 0.8143 1/1 [==============================] - 0s 1ms/step - loss: 1.2371 - accuracy: 0.6789 Testing accuracy after 160 epochs: 67.9% Epoch 1/20 18/18 [==============================] - 2s 116ms/step - loss: 0.6425 - accuracy: 0.8082 Epoch 2/20 18/18 [==============================] - 2s 114ms/step - loss: 0.8630 - accuracy: 0.7541 Epoch 3/20 18/18 [==============================] - 2s 113ms/step - loss: 0.8265 - accuracy: 0.7271 Epoch 4/20 18/18 [==============================] - 2s 114ms/step - loss: 0.7261 - accuracy: 0.7576 Epoch 5/20 18/18 [==============================] - 2s 114ms/step - loss: 0.6568 - accuracy: 0.7820 Epoch 6/20 18/18 [==============================] - 2s 115ms/step - loss: 0.6168 - accuracy: 0.7960 Epoch 7/20 18/18 [==============================] - 2s 117ms/step - loss: 0.6057 - accuracy: 0.7995 Epoch 8/20 18/18 [==============================] - 2s 115ms/step - loss: 0.5472 - accuracy: 0.8239 Epoch 9/20 18/18 [==============================] - 2s 113ms/step - loss: 0.5612 - accuracy: 0.8108 Epoch 10/20 18/18 [==============================] - 2s 114ms/step - loss: 0.5000 - accuracy: 0.8405 Epoch 11/20 18/18 [==============================] - 2s 116ms/step - loss: 0.5409 - accuracy: 0.8300 Epoch 12/20 18/18 [==============================] - 2s 114ms/step - loss: 0.5724 - accuracy: 0.8117 Epoch 13/20 18/18 [==============================] - 2s 113ms/step - loss: 0.5376 - accuracy: 0.8248 Epoch 14/20 18/18 [==============================] - 2s 114ms/step - loss: 0.5843 - accuracy: 0.7934 Epoch 15/20 18/18 [==============================] - 2s 119ms/step - loss: 0.5269 - accuracy: 0.8248 Epoch 16/20 18/18 [==============================] - 2s 136ms/step - loss: 0.4892 - accuracy: 0.8492 Epoch 17/20 18/18 [==============================] - 2s 119ms/step - loss: 0.4783 - accuracy: 0.8396 Epoch 18/20 18/18 [==============================] - 2s 116ms/step - loss: 0.5136 - accuracy: 0.8405 Epoch 19/20 18/18 [==============================] - 2s 117ms/step - loss: 0.4613 - accuracy: 0.8457 Epoch 20/20 18/18 [==============================] - 2s 123ms/step - loss: 0.4444 - accuracy: 0.8535 1/1 [==============================] - 0s 2ms/step - loss: 1.1809 - accuracy: 0.7258 Testing accuracy after 180 epochs: 72.6% Epoch 1/20 18/18 [==============================] - 2s 115ms/step - loss: 0.4296 - accuracy: 0.8622 Epoch 2/20 18/18 [==============================] - 2s 125ms/step - loss: 0.5622 - accuracy: 0.8213 Epoch 3/20 18/18 [==============================] - 2s 118ms/step - loss: 0.4883 - accuracy: 0.8361 Epoch 4/20 18/18 [==============================] - 2s 116ms/step - loss: 0.4114 - accuracy: 0.8692 Epoch 5/20 18/18 [==============================] - 2s 115ms/step - loss: 0.4728 - accuracy: 0.8474 Epoch 6/20 18/18 [==============================] - 2s 114ms/step - loss: 0.5357 - accuracy: 0.8300 Epoch 7/20 18/18 [==============================] - 2s 115ms/step - loss: 0.4753 - accuracy: 0.8518 Epoch 8/20 18/18 [==============================] - 2s 116ms/step - loss: 0.5083 - accuracy: 0.8335 Epoch 9/20 18/18 [==============================] - 2s 117ms/step - loss: 0.4574 - accuracy: 0.8553 Epoch 10/20 18/18 [==============================] - 2s 114ms/step - loss: 0.4477 - accuracy: 0.8509 Epoch 11/20 18/18 [==============================] - 2s 114ms/step - loss: 0.3872 - accuracy: 0.8710 Epoch 12/20 18/18 [==============================] - 2s 115ms/step - loss: 0.3767 - accuracy: 0.8806 Epoch 13/20 18/18 [==============================] - 2s 115ms/step - loss: 0.4948 - accuracy: 0.8518 Epoch 14/20 18/18 [==============================] - 2s 116ms/step - loss: 0.5815 - accuracy: 0.8282 Epoch 15/20 18/18 [==============================] - 2s 115ms/step - loss: 0.4828 - accuracy: 0.8466 Epoch 16/20 18/18 [==============================] - 2s 115ms/step - loss: 0.4241 - accuracy: 0.8588 Epoch 17/20 18/18 [==============================] - 2s 115ms/step - loss: 0.4984 - accuracy: 0.8431 Epoch 18/20 18/18 [==============================] - 2s 117ms/step - loss: 0.3685 - accuracy: 0.8945 Epoch 19/20 18/18 [==============================] - 2s 116ms/step - loss: 0.4810 - accuracy: 0.8561 Epoch 20/20 18/18 [==============================] - 2s 113ms/step - loss: 0.7044 - accuracy: 0.7977 1/1 [==============================] - 0s 1ms/step - loss: 1.1721 - accuracy: 0.6923 Testing accuracy after 200 epochs: 69.2% Epoch 1/20 18/18 [==============================] - 2s 115ms/step - loss: 0.5457 - accuracy: 0.8221 Epoch 2/20 18/18 [==============================] - 2s 114ms/step - loss: 0.4860 - accuracy: 0.8518 Epoch 3/20 18/18 [==============================] - 2s 116ms/step - loss: 0.4326 - accuracy: 0.8570 Epoch 4/20 18/18 [==============================] - 2s 121ms/step - loss: 0.3763 - accuracy: 0.8762 Epoch 5/20 18/18 [==============================] - 2s 115ms/step - loss: 0.4390 - accuracy: 0.8640 Epoch 6/20 18/18 [==============================] - 2s 117ms/step - loss: 0.4161 - accuracy: 0.8649 Epoch 7/20 18/18 [==============================] - 2s 116ms/step - loss: 0.3990 - accuracy: 0.8718 Epoch 8/20 18/18 [==============================] - 2s 116ms/step - loss: 0.3664 - accuracy: 0.8779 Epoch 9/20 18/18 [==============================] - 2s 115ms/step - loss: 0.3762 - accuracy: 0.8684 Epoch 10/20 18/18 [==============================] - 2s 116ms/step - loss: 0.4269 - accuracy: 0.8622 Epoch 11/20 18/18 [==============================] - 2s 116ms/step - loss: 0.3370 - accuracy: 0.8971 Epoch 12/20 18/18 [==============================] - 2s 115ms/step - loss: 0.3176 - accuracy: 0.9024 Epoch 13/20 18/18 [==============================] - 2s 114ms/step - loss: 0.3251 - accuracy: 0.9015 Epoch 14/20 18/18 [==============================] - 2s 114ms/step - loss: 0.3168 - accuracy: 0.8945 Epoch 15/20 18/18 [==============================] - 2s 117ms/step - loss: 0.2839 - accuracy: 0.9119 Epoch 16/20 18/18 [==============================] - 2s 119ms/step - loss: 0.2659 - accuracy: 0.9119 Epoch 17/20 18/18 [==============================] - 2s 116ms/step - loss: 0.2776 - accuracy: 0.9076 Epoch 18/20 18/18 [==============================] - 2s 116ms/step - loss: 0.3001 - accuracy: 0.9041 Epoch 19/20 18/18 [==============================] - 2s 116ms/step - loss: 0.3296 - accuracy: 0.9032 Epoch 20/20 18/18 [==============================] - 2s 114ms/step - loss: 0.3551 - accuracy: 0.8875 1/1 [==============================] - 0s 1ms/step - loss: 1.3045 - accuracy: 0.7358 Testing accuracy after 220 epochs: 73.6% Epoch 1/20 18/18 [==============================] - 2s 121ms/step - loss: 0.3080 - accuracy: 0.8945 Epoch 2/20 18/18 [==============================] - 2s 112ms/step - loss: 0.3842 - accuracy: 0.8901 Epoch 3/20 18/18 [==============================] - 3s 149ms/step - loss: 0.4126 - accuracy: 0.8710 Epoch 4/20 18/18 [==============================] - 3s 148ms/step - loss: 0.3547 - accuracy: 0.8884 Epoch 5/20 18/18 [==============================] - 3s 150ms/step - loss: 0.2818 - accuracy: 0.9111 Epoch 6/20 18/18 [==============================] - 3s 174ms/step - loss: 0.2629 - accuracy: 0.9268 Epoch 7/20 18/18 [==============================] - 3s 152ms/step - loss: 0.2906 - accuracy: 0.9111 Epoch 8/20 18/18 [==============================] - 2s 116ms/step - loss: 0.3031 - accuracy: 0.9015 Epoch 9/20 18/18 [==============================] - 2s 120ms/step - loss: 0.3542 - accuracy: 0.8832 Epoch 10/20 18/18 [==============================] - 2s 114ms/step - loss: 0.3271 - accuracy: 0.8997 Epoch 11/20 18/18 [==============================] - 2s 113ms/step - loss: 0.5144 - accuracy: 0.8518 Epoch 12/20 18/18 [==============================] - 2s 114ms/step - loss: 0.5230 - accuracy: 0.8378 Epoch 13/20 18/18 [==============================] - 2s 113ms/step - loss: 0.4236 - accuracy: 0.8745 Epoch 14/20 18/18 [==============================] - 2s 115ms/step - loss: 0.3989 - accuracy: 0.8884 Epoch 15/20 18/18 [==============================] - 2s 120ms/step - loss: 0.3687 - accuracy: 0.8814 Epoch 16/20 18/18 [==============================] - 3s 149ms/step - loss: 0.3163 - accuracy: 0.8997 Epoch 17/20 18/18 [==============================] - 3s 158ms/step - loss: 0.3357 - accuracy: 0.8945 Epoch 18/20 18/18 [==============================] - 2s 119ms/step - loss: 0.3585 - accuracy: 0.8945 Epoch 19/20 18/18 [==============================] - 2s 115ms/step - loss: 0.5361 - accuracy: 0.8422 Epoch 20/20 18/18 [==============================] - 2s 122ms/step - loss: 0.5348 - accuracy: 0.8335 1/1 [==============================] - 0s 2ms/step - loss: 1.4235 - accuracy: 0.6823 Testing accuracy after 240 epochs: 68.2% Epoch 1/20 18/18 [==============================] - 3s 145ms/step - loss: 0.6312 - accuracy: 0.8047 Epoch 2/20 18/18 [==============================] - 3s 150ms/step - loss: 0.5032 - accuracy: 0.8474 Epoch 3/20 18/18 [==============================] - 3s 169ms/step - loss: 0.4588 - accuracy: 0.8614 Epoch 4/20 18/18 [==============================] - 3s 162ms/step - loss: 0.4221 - accuracy: 0.8832 Epoch 5/20 18/18 [==============================] - 3s 149ms/step - loss: 0.3142 - accuracy: 0.9032 Epoch 6/20 18/18 [==============================] - 3s 173ms/step - loss: 0.3908 - accuracy: 0.8788 Epoch 7/20 18/18 [==============================] - 3s 145ms/step - loss: 0.4234 - accuracy: 0.8727 Epoch 8/20 18/18 [==============================] - 2s 136ms/step - loss: 0.3655 - accuracy: 0.8893 Epoch 9/20 18/18 [==============================] - 3s 157ms/step - loss: 0.3508 - accuracy: 0.9024 Epoch 10/20 18/18 [==============================] - 3s 146ms/step - loss: 0.3316 - accuracy: 0.8980 Epoch 11/20 18/18 [==============================] - 3s 166ms/step - loss: 0.3106 - accuracy: 0.9058 Epoch 12/20 18/18 [==============================] - 3s 156ms/step - loss: 0.3033 - accuracy: 0.9015 Epoch 13/20 18/18 [==============================] - 3s 152ms/step - loss: 0.2830 - accuracy: 0.9146 Epoch 14/20 18/18 [==============================] - 2s 117ms/step - loss: 0.2931 - accuracy: 0.9076 Epoch 15/20 18/18 [==============================] - 2s 116ms/step - loss: 0.2591 - accuracy: 0.9250 Epoch 16/20 18/18 [==============================] - 2s 115ms/step - loss: 0.3869 - accuracy: 0.9006 Epoch 17/20 18/18 [==============================] - 2s 116ms/step - loss: 0.4954 - accuracy: 0.8483 Epoch 18/20 18/18 [==============================] - 2s 115ms/step - loss: 0.3511 - accuracy: 0.8762 Epoch 19/20 18/18 [==============================] - 3s 139ms/step - loss: 0.2817 - accuracy: 0.9093 Epoch 20/20 18/18 [==============================] - 2s 123ms/step - loss: 0.2695 - accuracy: 0.9154 1/1 [==============================] - 0s 2ms/step - loss: 1.3146 - accuracy: 0.7458 Testing accuracy after 260 epochs: 74.6% Epoch 1/20 18/18 [==============================] - 3s 145ms/step - loss: 0.2668 - accuracy: 0.9119 Epoch 2/20 18/18 [==============================] - 2s 118ms/step - loss: 0.2560 - accuracy: 0.9154 Epoch 3/20 18/18 [==============================] - 2s 129ms/step - loss: 0.2322 - accuracy: 0.9207 Epoch 4/20 18/18 [==============================] - 3s 150ms/step - loss: 0.2521 - accuracy: 0.9163 Epoch 5/20 18/18 [==============================] - 3s 154ms/step - loss: 0.3949 - accuracy: 0.8788 Epoch 6/20 18/18 [==============================] - 3s 183ms/step - loss: 0.3254 - accuracy: 0.9032 Epoch 7/20 18/18 [==============================] - 2s 120ms/step - loss: 0.2781 - accuracy: 0.9111 Epoch 8/20 18/18 [==============================] - 2s 115ms/step - loss: 0.2564 - accuracy: 0.9154 Epoch 9/20 18/18 [==============================] - 2s 115ms/step - loss: 0.2680 - accuracy: 0.9172 Epoch 10/20 18/18 [==============================] - 3s 147ms/step - loss: 0.2493 - accuracy: 0.9285 Epoch 11/20 18/18 [==============================] - 3s 149ms/step - loss: 0.2358 - accuracy: 0.9303 Epoch 12/20 18/18 [==============================] - 2s 136ms/step - loss: 0.2073 - accuracy: 0.9372 Epoch 13/20 18/18 [==============================] - 3s 143ms/step - loss: 0.2034 - accuracy: 0.9364 Epoch 14/20 18/18 [==============================] - 2s 139ms/step - loss: 0.2181 - accuracy: 0.9355 Epoch 15/20 18/18 [==============================] - 2s 122ms/step - loss: 0.2117 - accuracy: 0.9337 Epoch 16/20 18/18 [==============================] - 3s 154ms/step - loss: 0.2339 - accuracy: 0.9294 Epoch 17/20 18/18 [==============================] - 3s 152ms/step - loss: 0.2059 - accuracy: 0.9355 Epoch 18/20 18/18 [==============================] - 3s 150ms/step - loss: 0.2267 - accuracy: 0.9241 Epoch 19/20 18/18 [==============================] - 3s 156ms/step - loss: 0.2101 - accuracy: 0.9364 Epoch 20/20 18/18 [==============================] - 3s 149ms/step - loss: 0.2392 - accuracy: 0.9241 1/1 [==============================] - 0s 1ms/step - loss: 1.4929 - accuracy: 0.7559 Testing accuracy after 280 epochs: 75.6% Epoch 1/20 18/18 [==============================] - 2s 134ms/step - loss: 0.1904 - accuracy: 0.9451 Epoch 2/20 18/18 [==============================] - 2s 122ms/step - loss: 0.2163 - accuracy: 0.9303 Epoch 3/20 18/18 [==============================] - 2s 120ms/step - loss: 0.2736 - accuracy: 0.9311 Epoch 4/20 18/18 [==============================] - 2s 130ms/step - loss: 0.3340 - accuracy: 0.8954 Epoch 5/20 18/18 [==============================] - 2s 115ms/step - loss: 0.2771 - accuracy: 0.9137 Epoch 6/20 18/18 [==============================] - 2s 130ms/step - loss: 0.2455 - accuracy: 0.9180 Epoch 7/20 18/18 [==============================] - 2s 119ms/step - loss: 0.2334 - accuracy: 0.9285 Epoch 8/20 18/18 [==============================] - 2s 116ms/step - loss: 0.2422 - accuracy: 0.9180 Epoch 9/20 18/18 [==============================] - 2s 116ms/step - loss: 0.3279 - accuracy: 0.8980 Epoch 10/20 18/18 [==============================] - 2s 128ms/step - loss: 0.3010 - accuracy: 0.9207 Epoch 11/20 18/18 [==============================] - 2s 118ms/step - loss: 0.3095 - accuracy: 0.9102 Epoch 12/20 18/18 [==============================] - 2s 122ms/step - loss: 0.3220 - accuracy: 0.9058 Epoch 13/20 18/18 [==============================] - 2s 130ms/step - loss: 0.2498 - accuracy: 0.9215 Epoch 14/20 18/18 [==============================] - 2s 116ms/step - loss: 0.2988 - accuracy: 0.9128 Epoch 15/20 18/18 [==============================] - 2s 135ms/step - loss: 0.3638 - accuracy: 0.8971 Epoch 16/20 18/18 [==============================] - 2s 120ms/step - loss: 0.2579 - accuracy: 0.9119 Epoch 17/20 18/18 [==============================] - 2s 114ms/step - loss: 0.2414 - accuracy: 0.9276 Epoch 18/20 18/18 [==============================] - 2s 120ms/step - loss: 0.1923 - accuracy: 0.9355 Epoch 19/20 18/18 [==============================] - 2s 120ms/step - loss: 0.1731 - accuracy: 0.9459 Epoch 20/20 18/18 [==============================] - 3s 145ms/step - loss: 0.3045 - accuracy: 0.9163 1/1 [==============================] - 0s 1ms/step - loss: 1.5787 - accuracy: 0.7224 Testing accuracy after 300 epochs: 72.2% The model trained for 280 epochs performed best on held out test data (0.8% accuracy)
# We see that training accuracy appears very high at ths point (90-95%) while testing accuracy is around 75%
# We also notice that test accuracy is roughly increasing with more training
# Therefore we launch a second round of training and testing, just starting with the latest model
for testing_index in range(number_of_models_to_test):
testing_index = testing_index # + number_of_models_to_test # to account for previously trained model indices
model.fit(train_X, one_hot_train_Y, epochs=epochs_between_tests, batch_size=input_batch_size)
_, accuracy = model.evaluate(test_X, one_hot_test_Y, batch_size=one_hot_test_Y.shape[0])
number_of_epochs_trained_for_model = (testing_index+1) * epochs_between_tests
epochs_trained.append(number_of_epochs_trained_for_model)
print("Testing accuracy after {} additional epochs: {:.1f}%\n\n".format(number_of_epochs_trained_for_model, 100*accuracy))
all_test_model_accuracies[testing_index] = accuracy
model_filepath = "{}/lstm_walker_recognition_model_{}.h5".format(folder, testing_index)
model.save(model_filepath)
model_filepaths.append(model_filepath)
max_model_index = np.argmax(all_test_model_accuracies)
best_model_filepath = model_filepaths[max_model_index]
best_model_test_accuracy = all_test_model_accuracies[max_model_index]
best_model_epochs_trained = epochs_trained[max_model_index]
print("\nThe model trained for {} additional epochs performed best on held out test data ({:.1f}% accuracy)".format(best_model_epochs_trained, 100*best_model_test_accuracy))
Epoch 1/20 18/18 [==============================] - 2s 122ms/step - loss: 0.2653 - accuracy: 0.9137 Epoch 2/20 18/18 [==============================] - 3s 146ms/step - loss: 0.2931 - accuracy: 0.9041 Epoch 3/20 18/18 [==============================] - 2s 119ms/step - loss: 0.2427 - accuracy: 0.9198 Epoch 4/20 18/18 [==============================] - 2s 124ms/step - loss: 0.3595 - accuracy: 0.8884 Epoch 5/20 18/18 [==============================] - 3s 141ms/step - loss: 0.2455 - accuracy: 0.9180 Epoch 6/20 18/18 [==============================] - 2s 117ms/step - loss: 0.2522 - accuracy: 0.9303 Epoch 7/20 18/18 [==============================] - 2s 116ms/step - loss: 0.2063 - accuracy: 0.9372 Epoch 8/20 18/18 [==============================] - 2s 120ms/step - loss: 0.2382 - accuracy: 0.9329 Epoch 9/20 18/18 [==============================] - 2s 119ms/step - loss: 0.2413 - accuracy: 0.9320 Epoch 10/20 18/18 [==============================] - 2s 119ms/step - loss: 0.2273 - accuracy: 0.9294 Epoch 11/20 18/18 [==============================] - 2s 115ms/step - loss: 0.2372 - accuracy: 0.9276 Epoch 12/20 18/18 [==============================] - 2s 116ms/step - loss: 0.2200 - accuracy: 0.9337 Epoch 13/20 18/18 [==============================] - 2s 115ms/step - loss: 0.2382 - accuracy: 0.9215 Epoch 14/20 18/18 [==============================] - 2s 120ms/step - loss: 0.1781 - accuracy: 0.9494 Epoch 15/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1636 - accuracy: 0.9468 Epoch 16/20 18/18 [==============================] - 2s 122ms/step - loss: 0.1820 - accuracy: 0.9416 Epoch 17/20 18/18 [==============================] - 2s 121ms/step - loss: 0.1550 - accuracy: 0.9520 Epoch 18/20 18/18 [==============================] - 2s 121ms/step - loss: 0.1614 - accuracy: 0.9451 Epoch 19/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1665 - accuracy: 0.9459 Epoch 20/20 18/18 [==============================] - 2s 114ms/step - loss: 0.1754 - accuracy: 0.9494 1/1 [==============================] - 0s 2ms/step - loss: 1.6940 - accuracy: 0.7492 Testing accuracy after 20 additional epochs: 74.9% Epoch 1/20 18/18 [==============================] - 2s 117ms/step - loss: 0.4720 - accuracy: 0.8910 Epoch 2/20 18/18 [==============================] - 2s 115ms/step - loss: 0.3330 - accuracy: 0.9119 Epoch 3/20 18/18 [==============================] - 2s 115ms/step - loss: 0.3294 - accuracy: 0.9006 Epoch 4/20 18/18 [==============================] - 2s 118ms/step - loss: 0.3366 - accuracy: 0.9085 Epoch 5/20 18/18 [==============================] - 2s 115ms/step - loss: 0.3554 - accuracy: 0.9015 Epoch 6/20 18/18 [==============================] - 2s 115ms/step - loss: 0.3336 - accuracy: 0.8989 Epoch 7/20 18/18 [==============================] - 2s 132ms/step - loss: 0.2703 - accuracy: 0.9146 Epoch 8/20 18/18 [==============================] - 2s 118ms/step - loss: 0.2237 - accuracy: 0.9337 Epoch 9/20 18/18 [==============================] - 2s 117ms/step - loss: 0.2057 - accuracy: 0.9398 Epoch 10/20 18/18 [==============================] - 2s 117ms/step - loss: 0.1550 - accuracy: 0.9555 Epoch 11/20 18/18 [==============================] - 2s 117ms/step - loss: 0.1929 - accuracy: 0.9372 Epoch 12/20 18/18 [==============================] - 2s 117ms/step - loss: 0.2344 - accuracy: 0.9250 Epoch 13/20 18/18 [==============================] - 2s 116ms/step - loss: 0.2028 - accuracy: 0.9364 Epoch 14/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1931 - accuracy: 0.9416 Epoch 15/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1423 - accuracy: 0.9520 Epoch 16/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1387 - accuracy: 0.9520 Epoch 17/20 18/18 [==============================] - 2s 117ms/step - loss: 0.1370 - accuracy: 0.9573 Epoch 18/20 18/18 [==============================] - 2s 116ms/step - loss: 0.1599 - accuracy: 0.9494 Epoch 19/20 18/18 [==============================] - 2s 115ms/step - loss: 0.3934 - accuracy: 0.9154 Epoch 20/20 18/18 [==============================] - 2s 114ms/step - loss: 0.6137 - accuracy: 0.8405 1/1 [==============================] - 0s 2ms/step - loss: 1.2670 - accuracy: 0.7057 Testing accuracy after 40 additional epochs: 70.6% Epoch 1/20 18/18 [==============================] - 2s 123ms/step - loss: 0.4121 - accuracy: 0.8701 Epoch 2/20 18/18 [==============================] - 2s 117ms/step - loss: 0.2409 - accuracy: 0.9137 Epoch 3/20 18/18 [==============================] - 2s 123ms/step - loss: 0.2081 - accuracy: 0.9285 Epoch 4/20 18/18 [==============================] - 2s 116ms/step - loss: 0.2078 - accuracy: 0.9372 Epoch 5/20 18/18 [==============================] - 2s 120ms/step - loss: 0.1731 - accuracy: 0.9468 Epoch 6/20 18/18 [==============================] - 2s 116ms/step - loss: 0.1444 - accuracy: 0.9555 Epoch 7/20 18/18 [==============================] - 2s 116ms/step - loss: 0.1458 - accuracy: 0.9512 Epoch 8/20 18/18 [==============================] - 2s 116ms/step - loss: 0.1444 - accuracy: 0.9599 Epoch 9/20 18/18 [==============================] - 2s 120ms/step - loss: 0.1312 - accuracy: 0.9564 Epoch 10/20 18/18 [==============================] - 2s 118ms/step - loss: 0.1307 - accuracy: 0.9582 Epoch 11/20 18/18 [==============================] - 2s 118ms/step - loss: 0.1359 - accuracy: 0.9564 Epoch 12/20 18/18 [==============================] - 2s 117ms/step - loss: 0.1308 - accuracy: 0.9555 Epoch 13/20 18/18 [==============================] - 2s 117ms/step - loss: 0.1270 - accuracy: 0.9634 Epoch 14/20 18/18 [==============================] - 3s 144ms/step - loss: 0.3530 - accuracy: 0.9119 Epoch 15/20 18/18 [==============================] - 2s 124ms/step - loss: 0.5290 - accuracy: 0.8544 Epoch 16/20 18/18 [==============================] - 2s 119ms/step - loss: 0.3789 - accuracy: 0.8779 Epoch 17/20 18/18 [==============================] - 2s 120ms/step - loss: 0.2999 - accuracy: 0.9024 Epoch 18/20 18/18 [==============================] - 2s 117ms/step - loss: 0.2957 - accuracy: 0.9041 Epoch 19/20 18/18 [==============================] - 2s 117ms/step - loss: 0.2324 - accuracy: 0.9259 Epoch 20/20 18/18 [==============================] - 2s 117ms/step - loss: 0.1775 - accuracy: 0.9425 1/1 [==============================] - 0s 1ms/step - loss: 1.4669 - accuracy: 0.7625 Testing accuracy after 60 additional epochs: 76.3% Epoch 1/20 18/18 [==============================] - 2s 123ms/step - loss: 0.1809 - accuracy: 0.9425 Epoch 2/20 18/18 [==============================] - 2s 118ms/step - loss: 0.2290 - accuracy: 0.9355 Epoch 3/20 18/18 [==============================] - 2s 121ms/step - loss: 0.4448 - accuracy: 0.8867 Epoch 4/20 18/18 [==============================] - 2s 119ms/step - loss: 0.3926 - accuracy: 0.8753 Epoch 5/20 18/18 [==============================] - 2s 119ms/step - loss: 0.2940 - accuracy: 0.9198 Epoch 6/20 18/18 [==============================] - 2s 117ms/step - loss: 0.3013 - accuracy: 0.9224 Epoch 7/20 18/18 [==============================] - 2s 120ms/step - loss: 0.4075 - accuracy: 0.8901 Epoch 8/20 18/18 [==============================] - 2s 119ms/step - loss: 0.3126 - accuracy: 0.9085 Epoch 9/20 18/18 [==============================] - 2s 117ms/step - loss: 0.2510 - accuracy: 0.9294 Epoch 10/20 18/18 [==============================] - 2s 120ms/step - loss: 0.2231 - accuracy: 0.9355 Epoch 11/20 18/18 [==============================] - 2s 117ms/step - loss: 0.1821 - accuracy: 0.9425 Epoch 12/20 18/18 [==============================] - 2s 120ms/step - loss: 0.1946 - accuracy: 0.9390 Epoch 13/20 18/18 [==============================] - 2s 117ms/step - loss: 0.1829 - accuracy: 0.9503 Epoch 14/20 18/18 [==============================] - 2s 118ms/step - loss: 0.1652 - accuracy: 0.9538 Epoch 15/20 18/18 [==============================] - 2s 126ms/step - loss: 0.1701 - accuracy: 0.9512 Epoch 16/20 18/18 [==============================] - 2s 120ms/step - loss: 0.1868 - accuracy: 0.9433 Epoch 17/20 18/18 [==============================] - 2s 117ms/step - loss: 0.1347 - accuracy: 0.9643 Epoch 18/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1549 - accuracy: 0.9555 Epoch 19/20 18/18 [==============================] - 2s 113ms/step - loss: 0.1436 - accuracy: 0.9608 Epoch 20/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1250 - accuracy: 0.9590 1/1 [==============================] - 0s 3ms/step - loss: 1.5178 - accuracy: 0.7692 Testing accuracy after 80 additional epochs: 76.9% Epoch 1/20 18/18 [==============================] - 2s 114ms/step - loss: 0.1335 - accuracy: 0.9582 Epoch 2/20 18/18 [==============================] - 2s 114ms/step - loss: 0.1241 - accuracy: 0.9643 Epoch 3/20 18/18 [==============================] - 2s 113ms/step - loss: 0.1531 - accuracy: 0.9555 Epoch 4/20 18/18 [==============================] - 2s 113ms/step - loss: 0.1524 - accuracy: 0.9547 Epoch 5/20 18/18 [==============================] - 2s 113ms/step - loss: 0.1366 - accuracy: 0.9538 Epoch 6/20 18/18 [==============================] - 2s 113ms/step - loss: 0.1495 - accuracy: 0.9520 Epoch 7/20 18/18 [==============================] - 2s 112ms/step - loss: 0.1353 - accuracy: 0.9564 Epoch 8/20 18/18 [==============================] - 2s 113ms/step - loss: 0.3099 - accuracy: 0.9241 Epoch 9/20 18/18 [==============================] - 2s 116ms/step - loss: 0.5383 - accuracy: 0.8701 Epoch 10/20 18/18 [==============================] - 2s 113ms/step - loss: 0.4158 - accuracy: 0.8858 Epoch 11/20 18/18 [==============================] - 2s 113ms/step - loss: 0.3272 - accuracy: 0.9050 Epoch 12/20 18/18 [==============================] - 2s 114ms/step - loss: 0.2184 - accuracy: 0.9294 Epoch 13/20 18/18 [==============================] - 2s 120ms/step - loss: 0.1883 - accuracy: 0.9416 Epoch 14/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1587 - accuracy: 0.9459 Epoch 15/20 18/18 [==============================] - 2s 114ms/step - loss: 0.1511 - accuracy: 0.9547 Epoch 16/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1228 - accuracy: 0.9616 Epoch 17/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1417 - accuracy: 0.9573 Epoch 18/20 18/18 [==============================] - 2s 113ms/step - loss: 0.1549 - accuracy: 0.9590 Epoch 19/20 18/18 [==============================] - 2s 113ms/step - loss: 0.1562 - accuracy: 0.9564 Epoch 20/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1860 - accuracy: 0.9486 1/1 [==============================] - 0s 2ms/step - loss: 1.7033 - accuracy: 0.7592 Testing accuracy after 100 additional epochs: 75.9% Epoch 1/20 18/18 [==============================] - 2s 114ms/step - loss: 0.2180 - accuracy: 0.9355 Epoch 2/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1489 - accuracy: 0.9608 Epoch 3/20 18/18 [==============================] - 2s 116ms/step - loss: 0.1711 - accuracy: 0.9494 Epoch 4/20 18/18 [==============================] - 2s 115ms/step - loss: 0.2529 - accuracy: 0.9329 Epoch 5/20 18/18 [==============================] - 2s 114ms/step - loss: 0.3105 - accuracy: 0.9154 Epoch 6/20 18/18 [==============================] - 2s 114ms/step - loss: 0.2234 - accuracy: 0.9303 Epoch 7/20 18/18 [==============================] - 3s 141ms/step - loss: 0.1948 - accuracy: 0.9372 Epoch 8/20 18/18 [==============================] - 2s 129ms/step - loss: 0.1723 - accuracy: 0.9512 Epoch 9/20 18/18 [==============================] - 2s 117ms/step - loss: 0.1558 - accuracy: 0.9564 Epoch 10/20 18/18 [==============================] - 2s 116ms/step - loss: 0.1316 - accuracy: 0.9564 Epoch 11/20 18/18 [==============================] - 2s 116ms/step - loss: 0.1442 - accuracy: 0.9520 Epoch 12/20 18/18 [==============================] - 2s 116ms/step - loss: 0.1333 - accuracy: 0.9573 Epoch 13/20 18/18 [==============================] - 2s 116ms/step - loss: 0.1055 - accuracy: 0.9686 Epoch 14/20 18/18 [==============================] - 2s 120ms/step - loss: 0.1175 - accuracy: 0.9677 Epoch 15/20 18/18 [==============================] - 2s 124ms/step - loss: 0.2069 - accuracy: 0.9425 Epoch 16/20 18/18 [==============================] - 2s 116ms/step - loss: 0.1753 - accuracy: 0.9459 Epoch 17/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1742 - accuracy: 0.9468 Epoch 18/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1590 - accuracy: 0.9503 Epoch 19/20 18/18 [==============================] - 2s 116ms/step - loss: 0.1694 - accuracy: 0.9486 Epoch 20/20 18/18 [==============================] - 2s 117ms/step - loss: 0.1498 - accuracy: 0.9442 1/1 [==============================] - 0s 2ms/step - loss: 1.6868 - accuracy: 0.7492 Testing accuracy after 120 additional epochs: 74.9% Epoch 1/20 18/18 [==============================] - 2s 118ms/step - loss: 0.1535 - accuracy: 0.9494 Epoch 2/20 18/18 [==============================] - 2s 115ms/step - loss: 0.4269 - accuracy: 0.9032 Epoch 3/20 18/18 [==============================] - 2s 115ms/step - loss: 0.3909 - accuracy: 0.8980 Epoch 4/20 18/18 [==============================] - 2s 116ms/step - loss: 0.3581 - accuracy: 0.8980 Epoch 5/20 18/18 [==============================] - 2s 116ms/step - loss: 0.2286 - accuracy: 0.9224 Epoch 6/20 18/18 [==============================] - 2s 114ms/step - loss: 0.2088 - accuracy: 0.9320 Epoch 7/20 18/18 [==============================] - 2s 115ms/step - loss: 0.6760 - accuracy: 0.8492 Epoch 8/20 18/18 [==============================] - 3s 152ms/step - loss: 1.3396 - accuracy: 0.6818 Epoch 9/20 18/18 [==============================] - 2s 117ms/step - loss: 0.7833 - accuracy: 0.7428 Epoch 10/20 18/18 [==============================] - 2s 115ms/step - loss: 0.5284 - accuracy: 0.8152 Epoch 11/20 18/18 [==============================] - 2s 115ms/step - loss: 0.4476 - accuracy: 0.8431 Epoch 12/20 18/18 [==============================] - 2s 124ms/step - loss: 0.3612 - accuracy: 0.8875 Epoch 13/20 18/18 [==============================] - 2s 115ms/step - loss: 0.3148 - accuracy: 0.8963 Epoch 14/20 18/18 [==============================] - 2s 115ms/step - loss: 0.2772 - accuracy: 0.9050 Epoch 15/20 18/18 [==============================] - 2s 115ms/step - loss: 0.2429 - accuracy: 0.9215 Epoch 16/20 18/18 [==============================] - 2s 113ms/step - loss: 0.2700 - accuracy: 0.9085 Epoch 17/20 18/18 [==============================] - 2s 115ms/step - loss: 0.2609 - accuracy: 0.9250 Epoch 18/20 18/18 [==============================] - 2s 113ms/step - loss: 0.2311 - accuracy: 0.9268 Epoch 19/20 18/18 [==============================] - 2s 112ms/step - loss: 0.2360 - accuracy: 0.9250 Epoch 20/20 18/18 [==============================] - 2s 114ms/step - loss: 0.2463 - accuracy: 0.9303 1/1 [==============================] - 0s 1ms/step - loss: 1.3512 - accuracy: 0.7659 Testing accuracy after 140 additional epochs: 76.6% Epoch 1/20 18/18 [==============================] - 2s 117ms/step - loss: 0.2132 - accuracy: 0.9241 Epoch 2/20 18/18 [==============================] - 2s 114ms/step - loss: 0.2053 - accuracy: 0.9381 Epoch 3/20 18/18 [==============================] - 2s 116ms/step - loss: 0.1722 - accuracy: 0.9425 Epoch 4/20 18/18 [==============================] - 2s 114ms/step - loss: 0.1778 - accuracy: 0.9425 Epoch 5/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1511 - accuracy: 0.9555 Epoch 6/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1785 - accuracy: 0.9364 Epoch 7/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1344 - accuracy: 0.9616 Epoch 8/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1417 - accuracy: 0.9529 Epoch 9/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1404 - accuracy: 0.9555 Epoch 10/20 18/18 [==============================] - 2s 116ms/step - loss: 0.1403 - accuracy: 0.9520 Epoch 11/20 18/18 [==============================] - 2s 113ms/step - loss: 0.1204 - accuracy: 0.9608 Epoch 12/20 18/18 [==============================] - 2s 120ms/step - loss: 0.1076 - accuracy: 0.9651 Epoch 13/20 18/18 [==============================] - 2s 114ms/step - loss: 0.1020 - accuracy: 0.9660 Epoch 14/20 18/18 [==============================] - 2s 116ms/step - loss: 0.0979 - accuracy: 0.9730 Epoch 15/20 18/18 [==============================] - 2s 117ms/step - loss: 0.1199 - accuracy: 0.9590 Epoch 16/20 18/18 [==============================] - 2s 113ms/step - loss: 0.1118 - accuracy: 0.9634 Epoch 17/20 18/18 [==============================] - 2s 115ms/step - loss: 0.2261 - accuracy: 0.9477 Epoch 18/20 18/18 [==============================] - 2s 113ms/step - loss: 0.1801 - accuracy: 0.9494 Epoch 19/20 18/18 [==============================] - 2s 113ms/step - loss: 0.1532 - accuracy: 0.9520 Epoch 20/20 18/18 [==============================] - 3s 143ms/step - loss: 0.2394 - accuracy: 0.9268 1/1 [==============================] - 0s 1ms/step - loss: 1.4673 - accuracy: 0.7659 Testing accuracy after 160 additional epochs: 76.6% Epoch 1/20 18/18 [==============================] - 2s 120ms/step - loss: 0.1924 - accuracy: 0.9486 Epoch 2/20 18/18 [==============================] - 2s 114ms/step - loss: 0.1182 - accuracy: 0.9634 Epoch 3/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1960 - accuracy: 0.9477 Epoch 4/20 18/18 [==============================] - 2s 114ms/step - loss: 0.5392 - accuracy: 0.8867 Epoch 5/20 18/18 [==============================] - 2s 113ms/step - loss: 0.4923 - accuracy: 0.8596 Epoch 6/20 18/18 [==============================] - 2s 115ms/step - loss: 0.3099 - accuracy: 0.9111 Epoch 7/20 18/18 [==============================] - 2s 119ms/step - loss: 0.2821 - accuracy: 0.9085 Epoch 8/20 18/18 [==============================] - 2s 113ms/step - loss: 0.2945 - accuracy: 0.9050 Epoch 9/20 18/18 [==============================] - 2s 126ms/step - loss: 0.2454 - accuracy: 0.9241 Epoch 10/20 18/18 [==============================] - 2s 114ms/step - loss: 0.2056 - accuracy: 0.9442 Epoch 11/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1816 - accuracy: 0.9459 Epoch 12/20 18/18 [==============================] - 2s 115ms/step - loss: 0.2073 - accuracy: 0.9337 Epoch 13/20 18/18 [==============================] - 2s 117ms/step - loss: 0.1898 - accuracy: 0.9381 Epoch 14/20 18/18 [==============================] - 2s 125ms/step - loss: 0.2211 - accuracy: 0.9459 Epoch 15/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1724 - accuracy: 0.9433 Epoch 16/20 18/18 [==============================] - 2s 116ms/step - loss: 0.2146 - accuracy: 0.9459 Epoch 17/20 18/18 [==============================] - 2s 116ms/step - loss: 0.2113 - accuracy: 0.9329 Epoch 18/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1631 - accuracy: 0.9538 Epoch 19/20 18/18 [==============================] - 2s 114ms/step - loss: 0.1445 - accuracy: 0.9564 Epoch 20/20 18/18 [==============================] - 2s 114ms/step - loss: 0.1252 - accuracy: 0.9616 1/1 [==============================] - 0s 1ms/step - loss: 1.5313 - accuracy: 0.7726 Testing accuracy after 180 additional epochs: 77.3% Epoch 1/20 18/18 [==============================] - 2s 116ms/step - loss: 0.1175 - accuracy: 0.9625 Epoch 2/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1323 - accuracy: 0.9634 Epoch 3/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1048 - accuracy: 0.9677 Epoch 4/20 18/18 [==============================] - 2s 114ms/step - loss: 0.0968 - accuracy: 0.9712 Epoch 5/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1207 - accuracy: 0.9599 Epoch 6/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1292 - accuracy: 0.9608 Epoch 7/20 18/18 [==============================] - 2s 123ms/step - loss: 0.1522 - accuracy: 0.9643 Epoch 8/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1983 - accuracy: 0.9477 Epoch 9/20 18/18 [==============================] - 2s 114ms/step - loss: 0.1507 - accuracy: 0.9573 Epoch 10/20 18/18 [==============================] - 2s 117ms/step - loss: 0.1525 - accuracy: 0.9538 Epoch 11/20 18/18 [==============================] - 2s 117ms/step - loss: 0.1190 - accuracy: 0.9599 Epoch 12/20 18/18 [==============================] - 2s 119ms/step - loss: 0.1181 - accuracy: 0.9590 Epoch 13/20 18/18 [==============================] - 2s 115ms/step - loss: 0.0940 - accuracy: 0.9651 Epoch 14/20 18/18 [==============================] - 2s 115ms/step - loss: 0.0802 - accuracy: 0.9782 Epoch 15/20 18/18 [==============================] - 2s 114ms/step - loss: 0.1019 - accuracy: 0.9651 Epoch 16/20 18/18 [==============================] - 2s 116ms/step - loss: 0.1446 - accuracy: 0.9625 Epoch 17/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1408 - accuracy: 0.9625 Epoch 18/20 18/18 [==============================] - 2s 113ms/step - loss: 0.1404 - accuracy: 0.9555 Epoch 19/20 18/18 [==============================] - 2s 114ms/step - loss: 0.1262 - accuracy: 0.9599 Epoch 20/20 18/18 [==============================] - 2s 117ms/step - loss: 0.2231 - accuracy: 0.9486 1/1 [==============================] - 0s 2ms/step - loss: 1.5303 - accuracy: 0.7860 Testing accuracy after 200 additional epochs: 78.6% Epoch 1/20 18/18 [==============================] - 2s 116ms/step - loss: 0.1953 - accuracy: 0.9512 Epoch 2/20 18/18 [==============================] - 2s 117ms/step - loss: 0.1477 - accuracy: 0.9590 Epoch 3/20 18/18 [==============================] - 2s 124ms/step - loss: 0.1226 - accuracy: 0.9660 Epoch 4/20 18/18 [==============================] - 2s 117ms/step - loss: 0.1641 - accuracy: 0.9547 Epoch 5/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1381 - accuracy: 0.9582 Epoch 6/20 18/18 [==============================] - 3s 142ms/step - loss: 0.1076 - accuracy: 0.9651 Epoch 7/20 18/18 [==============================] - 3s 149ms/step - loss: 0.1074 - accuracy: 0.9669 Epoch 8/20 18/18 [==============================] - 3s 155ms/step - loss: 0.0840 - accuracy: 0.9730 Epoch 9/20 18/18 [==============================] - 3s 149ms/step - loss: 0.0988 - accuracy: 0.9712 Epoch 10/20 18/18 [==============================] - 3s 149ms/step - loss: 0.1146 - accuracy: 0.9677 Epoch 11/20 18/18 [==============================] - 3s 148ms/step - loss: 0.1141 - accuracy: 0.9660 Epoch 12/20 18/18 [==============================] - 3s 178ms/step - loss: 0.0890 - accuracy: 0.9730 Epoch 13/20 18/18 [==============================] - 3s 157ms/step - loss: 0.0916 - accuracy: 0.9686 Epoch 14/20 18/18 [==============================] - 2s 137ms/step - loss: 0.0988 - accuracy: 0.9677 Epoch 15/20 18/18 [==============================] - 2s 130ms/step - loss: 0.1090 - accuracy: 0.9730 Epoch 16/20 18/18 [==============================] - 2s 136ms/step - loss: 0.0870 - accuracy: 0.9686 Epoch 17/20 18/18 [==============================] - 2s 136ms/step - loss: 0.1155 - accuracy: 0.9660 Epoch 18/20 18/18 [==============================] - 2s 132ms/step - loss: 0.1740 - accuracy: 0.9651 Epoch 19/20 18/18 [==============================] - 2s 132ms/step - loss: 0.2875 - accuracy: 0.9224 Epoch 20/20 18/18 [==============================] - 2s 130ms/step - loss: 0.2921 - accuracy: 0.9154 1/1 [==============================] - 0s 3ms/step - loss: 1.4087 - accuracy: 0.7492 Testing accuracy after 220 additional epochs: 74.9% Epoch 1/20 18/18 [==============================] - 2s 133ms/step - loss: 0.2358 - accuracy: 0.9346 Epoch 2/20 18/18 [==============================] - 2s 132ms/step - loss: 0.1797 - accuracy: 0.9407 Epoch 3/20 18/18 [==============================] - 2s 126ms/step - loss: 0.1799 - accuracy: 0.9398 Epoch 4/20 18/18 [==============================] - 2s 129ms/step - loss: 0.1457 - accuracy: 0.9590 Epoch 5/20 18/18 [==============================] - 2s 118ms/step - loss: 0.1936 - accuracy: 0.9468 Epoch 6/20 18/18 [==============================] - 2s 118ms/step - loss: 0.1397 - accuracy: 0.9538 Epoch 7/20 18/18 [==============================] - 2s 119ms/step - loss: 0.1254 - accuracy: 0.9529 Epoch 8/20 18/18 [==============================] - 2s 118ms/step - loss: 0.1023 - accuracy: 0.9704 Epoch 9/20 18/18 [==============================] - 2s 123ms/step - loss: 0.0932 - accuracy: 0.9721 Epoch 10/20 18/18 [==============================] - 2s 123ms/step - loss: 0.1044 - accuracy: 0.9660 Epoch 11/20 18/18 [==============================] - 2s 119ms/step - loss: 0.0895 - accuracy: 0.9695 Epoch 12/20 18/18 [==============================] - 2s 124ms/step - loss: 0.1011 - accuracy: 0.9643 Epoch 13/20 18/18 [==============================] - 2s 115ms/step - loss: 0.0777 - accuracy: 0.9773 Epoch 14/20 18/18 [==============================] - 2s 114ms/step - loss: 0.0924 - accuracy: 0.9695 Epoch 15/20 18/18 [==============================] - 2s 114ms/step - loss: 0.0729 - accuracy: 0.9756 Epoch 16/20 18/18 [==============================] - 2s 118ms/step - loss: 0.0978 - accuracy: 0.9660 Epoch 17/20 18/18 [==============================] - 2s 116ms/step - loss: 0.0983 - accuracy: 0.9686 Epoch 18/20 18/18 [==============================] - 2s 116ms/step - loss: 0.0812 - accuracy: 0.9782 Epoch 19/20 18/18 [==============================] - 2s 116ms/step - loss: 0.0845 - accuracy: 0.9730 Epoch 20/20 18/18 [==============================] - 2s 117ms/step - loss: 0.0940 - accuracy: 0.9721 1/1 [==============================] - 0s 2ms/step - loss: 1.6166 - accuracy: 0.7659 Testing accuracy after 240 additional epochs: 76.6% Epoch 1/20 18/18 [==============================] - 2s 117ms/step - loss: 0.0983 - accuracy: 0.9704 Epoch 2/20 18/18 [==============================] - 2s 116ms/step - loss: 0.1088 - accuracy: 0.9651 Epoch 3/20 18/18 [==============================] - 2s 114ms/step - loss: 0.1896 - accuracy: 0.9599 Epoch 4/20 18/18 [==============================] - 2s 114ms/step - loss: 0.1741 - accuracy: 0.9529 Epoch 5/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1316 - accuracy: 0.9599 Epoch 6/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1243 - accuracy: 0.9599 Epoch 7/20 18/18 [==============================] - 2s 114ms/step - loss: 0.1744 - accuracy: 0.9477 Epoch 8/20 18/18 [==============================] - 2s 117ms/step - loss: 0.1853 - accuracy: 0.9477 Epoch 9/20 18/18 [==============================] - 2s 116ms/step - loss: 0.1744 - accuracy: 0.9442 Epoch 10/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1350 - accuracy: 0.9459 Epoch 11/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1433 - accuracy: 0.9573 Epoch 12/20 18/18 [==============================] - 2s 114ms/step - loss: 0.1689 - accuracy: 0.9477 Epoch 13/20 18/18 [==============================] - 2s 114ms/step - loss: 0.1672 - accuracy: 0.9494 Epoch 14/20 18/18 [==============================] - 2s 119ms/step - loss: 0.1960 - accuracy: 0.9477 Epoch 15/20 18/18 [==============================] - 2s 115ms/step - loss: 0.2420 - accuracy: 0.9346 Epoch 16/20 18/18 [==============================] - 2s 120ms/step - loss: 0.1372 - accuracy: 0.9477 Epoch 17/20 18/18 [==============================] - 3s 152ms/step - loss: 0.0994 - accuracy: 0.9669 Epoch 18/20 18/18 [==============================] - 3s 149ms/step - loss: 0.0832 - accuracy: 0.9730 Epoch 19/20 18/18 [==============================] - 3s 172ms/step - loss: 0.0902 - accuracy: 0.9686 Epoch 20/20 18/18 [==============================] - 3s 164ms/step - loss: 0.0658 - accuracy: 0.9817 1/1 [==============================] - 0s 1ms/step - loss: 1.6104 - accuracy: 0.7692 Testing accuracy after 260 additional epochs: 76.9% Epoch 1/20 18/18 [==============================] - 3s 146ms/step - loss: 0.0918 - accuracy: 0.9730 Epoch 2/20 18/18 [==============================] - 2s 119ms/step - loss: 0.0783 - accuracy: 0.9730 Epoch 3/20 18/18 [==============================] - 2s 115ms/step - loss: 0.0715 - accuracy: 0.9808 Epoch 4/20 18/18 [==============================] - 2s 115ms/step - loss: 0.0661 - accuracy: 0.9791 Epoch 5/20 18/18 [==============================] - 2s 117ms/step - loss: 0.0700 - accuracy: 0.9791 Epoch 6/20 18/18 [==============================] - 2s 116ms/step - loss: 0.0786 - accuracy: 0.9721 Epoch 7/20 18/18 [==============================] - 2s 125ms/step - loss: 0.0601 - accuracy: 0.9765 Epoch 8/20 18/18 [==============================] - 2s 120ms/step - loss: 0.0604 - accuracy: 0.9817 Epoch 9/20 18/18 [==============================] - 2s 116ms/step - loss: 0.0859 - accuracy: 0.9773 Epoch 10/20 18/18 [==============================] - 2s 125ms/step - loss: 0.0954 - accuracy: 0.9712 Epoch 11/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1484 - accuracy: 0.9608 Epoch 12/20 18/18 [==============================] - 2s 118ms/step - loss: 0.2171 - accuracy: 0.9486 Epoch 13/20 18/18 [==============================] - 2s 116ms/step - loss: 0.4613 - accuracy: 0.8736 Epoch 14/20 18/18 [==============================] - 2s 118ms/step - loss: 0.3674 - accuracy: 0.9024 Epoch 15/20 18/18 [==============================] - 2s 117ms/step - loss: 0.3646 - accuracy: 0.9041 Epoch 16/20 18/18 [==============================] - 2s 115ms/step - loss: 0.2455 - accuracy: 0.9198 Epoch 17/20 18/18 [==============================] - 2s 114ms/step - loss: 0.1767 - accuracy: 0.9372 Epoch 18/20 18/18 [==============================] - 2s 122ms/step - loss: 0.1245 - accuracy: 0.9590 Epoch 19/20 18/18 [==============================] - 2s 120ms/step - loss: 0.1304 - accuracy: 0.9520 Epoch 20/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1239 - accuracy: 0.9564 1/1 [==============================] - 0s 2ms/step - loss: 1.7474 - accuracy: 0.7559 Testing accuracy after 280 additional epochs: 75.6% Epoch 1/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1164 - accuracy: 0.9669 Epoch 2/20 18/18 [==============================] - 2s 115ms/step - loss: 0.0895 - accuracy: 0.9730 Epoch 3/20 18/18 [==============================] - 2s 115ms/step - loss: 0.0999 - accuracy: 0.9669 Epoch 4/20 18/18 [==============================] - 2s 115ms/step - loss: 0.0991 - accuracy: 0.9738 Epoch 5/20 18/18 [==============================] - 2s 117ms/step - loss: 0.1021 - accuracy: 0.9651 Epoch 6/20 18/18 [==============================] - 2s 117ms/step - loss: 0.1009 - accuracy: 0.9677 Epoch 7/20 18/18 [==============================] - 2s 121ms/step - loss: 0.0917 - accuracy: 0.9677 Epoch 8/20 18/18 [==============================] - 2s 126ms/step - loss: 0.0805 - accuracy: 0.9677 Epoch 9/20 18/18 [==============================] - 2s 116ms/step - loss: 0.0667 - accuracy: 0.9765 Epoch 10/20 18/18 [==============================] - 2s 115ms/step - loss: 0.0850 - accuracy: 0.9773 Epoch 11/20 18/18 [==============================] - 2s 116ms/step - loss: 0.0798 - accuracy: 0.9765 Epoch 12/20 18/18 [==============================] - 2s 114ms/step - loss: 0.0738 - accuracy: 0.9765 Epoch 13/20 18/18 [==============================] - 2s 114ms/step - loss: 0.2077 - accuracy: 0.9433 Epoch 14/20 18/18 [==============================] - 2s 114ms/step - loss: 0.1878 - accuracy: 0.9451 Epoch 15/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1562 - accuracy: 0.9529 Epoch 16/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1131 - accuracy: 0.9669 Epoch 17/20 18/18 [==============================] - 2s 115ms/step - loss: 0.1254 - accuracy: 0.9677 Epoch 18/20 18/18 [==============================] - 2s 133ms/step - loss: 0.1290 - accuracy: 0.9564 Epoch 19/20 18/18 [==============================] - 3s 145ms/step - loss: 0.1236 - accuracy: 0.9590 Epoch 20/20 18/18 [==============================] - 3s 140ms/step - loss: 0.0925 - accuracy: 0.9721 1/1 [==============================] - 0s 1ms/step - loss: 1.9642 - accuracy: 0.7559 Testing accuracy after 300 additional epochs: 75.6% The model trained for 200 additional epochs performed best on held out test data (78.6% accuracy)
# Now we see that there is no longer a clear benefit to further training on this architecture.
# Final best testing result is 78%.
# The input data (including test data) still has several noisy sequences of data that have not been cleaned (e.g. dead signals).
# In other words, there are many sequences in the test data where there are no patterns to pick up, because of noise / chaos in walking data.
# Therefore, 78% test result, for a basic vanilla architecture, seems like a good starting point for further refinement of both data and architecture.
# For further experimental validation, we could cross-validate the test data set with N-folds.
# As well, given this initial result, we could begin to do a more rigorous architectural and hyperparameter search of the LSTM neural network.
# For cleaning data, from visual analysis of it, the following could be explored:
# (a) remove corrupt data, e.g. "dead" periods, where sensor "drifts", reading approximately the same acceleration value (data that is impossible to label)
# (b) remove data where user is not clearly walking forward, e.g. where z_acceleration is not high enough, based on a threshold per user